# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/topics/item-pipeline.html
from scrapy.contrib.exporter import JsonLinesItemExporter, CsvItemExporter
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
import  os

if not os.path.exists("results"):
    os.mkdir("results")
    
class CSVPipeline(object):
    
    def __init__(self):
        self.files = {}
        self.csvs = {}
        dispatcher.connect(self.spider_opened, signals.spider_opened)
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def spider_opened(self, spider):
        self.files[spider] = {}
        filename = os.path.join("results", "%s.csv" %spider.name)
        self.files[spider] = open(filename, 'wb') 
        self.csvs[spider] = CsvItemExporter(self.files[spider])
        self.csvs[spider].start_exporting()
        self.files[spider].flush()

    def spider_closed(self, spider):
        self.csvs[spider].finish_exporting()
        self.files[spider].close()
        del self.files[spider]
        del self.csvs[spider]
        
    def process_item(self, item, spider):
        for field in item:
            if item[field]: item[field] = item[field][0].strip()
            else: item[field] = u''
        try:
            print item['title'].encode()
        except:pass

        self.csvs[spider].export_item(item)
        return item
    
class JsonPipeline(object):
    def __init__(self):
        self.files = {}
        self.jsons = {}
        dispatcher.connect(self.spider_opened, signals.spider_opened)
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def spider_opened(self, spider):
        self.files[spider] = {}
        filename = os.path.join("results", "%s.json" %spider.name)
        self.files[spider] = open(filename, 'wb') 
        self.jsons[spider] = JsonLinesItemExporter(self.files[spider], 
                                                   export_empty_fields=True)
        self.jsons[spider].start_exporting()

    def spider_closed(self, spider):
        self.jsons[spider].finish_exporting()
        self.files[spider].close()
        del self.files[spider]
        del self.jsons[spider]
    def process_item(self, item, spider):
        self.jsons[spider].export_item(item)
        return item
